# -*- coding: utf-8 -*-

import commonlib
import workerpool
import os, traceback, datetime, time
import logging
import download
import pprint
from MongoModel import MongoModel
from pymongo.objectid import ObjectId


DOWNLOAD_PATH = '/home/hoangnamhai/HarvestedData/nhaccuatui/'
PREFIX = '/uploads/nhaccuatui/'
MONGO_SERVER = 'beta.mana.vn'    # Server Mana 64bit
MONGO_PORT = 27017
TYM_DB = 'my_database'
ARTICLE_COLLECTION = 'nct_article'

logging.basicConfig()
logger = logging.getLogger('nhaccuatui_tym')
logger.setLevel(logging.DEBUG)

class NCTTym(MongoModel):
    
    categories = {
          'nhactre': 'Nhạc Trẻ',
          'trutinh': 'Nhạc Trữ tình',
          'cachmang': 'Nhạc Cách mạng',
          'khongloi': 'Nhạc Không lời',
          'hiphop': 'Rock, Hip Hop Việt',
          'hoa': 'Nhạc Hoa',
          'nhat': 'Nhạc Nhật',
          'han': 'Nhạc Hàn Quốc',
          'thieunhi': 'Nhạc Thiếu Nhi',
          'anh': 'Nhạc US - UK',
          'khac': 'Thể Loại Khác',
          'tienchien': 'Nhạc Tiền Chiến'
          }
    qUrl = 'http://www.nhaccuatui.com/tim_kiem?g={0}&page={1}&t={2}'
    sUrl = 'http://www.nhaccuatui.com/nghe?M={0}'
    
    def __init__(self, host='localhost', port=27017):
        MongoModel.__init__(self, host, port)
        self.authorId = self.findCrawlerAuthorId()
        
    def __del__(self):
        MongoModel.__del__(self)
    
    def getLeftRootOfCategory(self, catName):
        db = self.connection[TYM_DB]
        category_types = db['categorytype']
        type = category_types.find_one({'name': 'mp3'}, {'_id': 1})
        if type==None: return None
        collection = db['category']
        q = collection.find_one({'data': catName, 'type_id': type['_id']}, {'lft': 1, 'root_id': 1})
        if q: return {'category': q['lft'], 'root': q['root_id']}
        return None
        
    def findCrawlerAuthorId(self, name='crawler'):
        logger.info('call findCrawlerAuthorId')
        db = self.connection[TYM_DB]
        collection = db['backend_user']
        cr = collection.find_one({'username': 'crawler'}, {'_id': 1})
        if cr is not None:
            return ObjectId(cr['_id'])
        return ''
            
    def isExists(self, songId):
        db = self.connection[TYM_DB]
        collection = db[ARTICLE_COLLECTION]
        if collection.find_one({'songId': commonlib.getMD5Hash(songId)}):
            return True
        return False
    
    def processPage(self, catId, page=1):
        logger.debug('call processPage({0}, {1})'.format(catId, page))
        url = self.qUrl.format(catId, page, time.time())
        try:
            tree = commonlib.getXMLTree(url)
            if tree == '': return ''
            data = []
            for elem in tree.xpath("//div[@class='grid-music']//div[@class='item']"):
                songId = commonlib.getAttribText(elem, 'id')
                if not self.isExists(songId):
                    data.append(songId)
            for songId in data:
#                print songId
                self.processSong(songId, catId)
        except:
            logger.error(traceback.format_exc())
    
    def download(self, url):
        logger.debug('call download with url: {0}'.format(url))
        filesize, filename = download.downloadNCT(url, DOWNLOAD_PATH, PREFIX)
        return filesize, filename
    
    def processSong(self, songId, catId):
        url = self.sUrl.format(songId)
        logger.debug('processSong (%s)' % url)
        try:
            tree = commonlib.getXMLTree(url)
            embedFlash = tree.xpath("//object[@id='flash-player']/param[@name='flashvars']")
            if len(embedFlash) > 0:
                value = commonlib.getAttribText(embedFlash, 'value')
                link = commonlib.extractWithRegEx(r'file=(.+)', value, 1)
                data = self.parseNCTXML(link)
                for song in data:
                    fileSize, fileName = self.download(song['tmpLink'])
                    if fileName.startswith('Err') and fileSize != '-1': continue
                    self.save(catId, {
                          'title': song['title'], 'artist': song['artist'],
                          'original_name': os.path.basename(song['tmpLink']), 
                          'size': fileSize,
                          'source': fileName,
                          'songId': commonlib.getMD5Hash(songId),
                          })
        except:
            logger.error(traceback.format_exc())
        
    def save(self, catId, args):
        logger.debug('save to mongodb, parameter: catId={0}, args={1}'.format(catId, args))
        db = self.connection[TYM_DB]
        collection = db[ARTICLE_COLLECTION]
        cat = self.getLeftRootOfCategory(self.categories[catId])
        if cat == None: return ''
        if collection.find_one({'songId': args['songId']}): return
        data = {
            'author_id': self.authorId,
            'category': cat['category'],
            'root': cat['root'],
            'title': args['title'],
            'songId': args['songId'],
            'type': 'mp3',
            'source': 'nhaccuatui.com',
            'content': {
                'intro': '{0} - {1}'.format(args['title'], args['artist']),
                'preview': '',
                'download': {
                     'original_name': args['original_name'],
                     'size': args['size'],
                     'source': args['source'],
                     'singer': args['artist']
                     }
                },
            'is_active': True,
            'created_at': datetime.datetime.utcnow(),
            'date': datetime.datetime.utcnow()
            }
        collection.save(data)
        
    def parseNCTXML(self, url):
        data = []
        try:
            tree = commonlib.getXMLTree(url, isXML=True)
            if tree == '': return []
            for elem in tree.xpath("//track"):
                song = {}
                song['artist'] = commonlib.getElementText(elem.xpath(".//creator"))
                song['title'] = commonlib.getElementText(elem.xpath(".//title"))
                song['tmpLink'] = commonlib.getElementText(elem.xpath(".//location"))
                data.append(song)
        except:
            logger.error(traceback.format_exc())
        pprint.pprint(data)
        return data
    
    def process(self, catId):
        logger.debug(unicode('call process %s' % self.categories[catId], 'utf8'))
        cat = self.getLeftRootOfCategory(self.categories[catId])
        if cat == None: 
            logger.warning(unicode('category %s doesn\'t exist' % self.categories[catId], 'utf8'))
            return
        # NOTE: khi ket qua den trang 26 thi nhac cua tui ko hien thi ket qua search nua
        for ipage in range(1, 26):
            self.processPage(catId, ipage)
    
if __name__ == '__main__':
    logger.info('start crawler nhaccuatui for tym')
    nct = NCTTym(MONGO_SERVER, MONGO_PORT)
    if nct.authorId == '':
        logger.info("user crawler doesn't exist, please create an username: crawler") 
        os._exit(1)
    try:
        pool = workerpool.WorkerPool(size=5)
        pool.map(nct.process, nct.categories.keys())
        pool.shutdown()
        pool.wait()
    except:
        logger.error(traceback.format_exc())
    logger.info('finished crawler nhaccuatui for tym at %s' % str(datetime.datetime.now()))
    pid = os.getpid()
    os._exit(1)
    os.kill(pid, 9)
    
        
